/*
 * anima/x86_64/pdpm.hh - PDP mappers.
 * Copyright (C) 2007  Mikhail Vorozhtsov
 * See the LICENSE section of the README file for details.
 */

/* $Id$ */

#ifndef HEADER_ANIMA_X86_64_PDPM_H
#define HEADER_ANIMA_X86_64_PDPM_H

#include HEADER (anima,pdb.hh)
#include HEADER (anima,locks.hh)
#include HEADER (utils,memory.hh)
#include HEADER (utils,setters.hh)

START_NAMESPACE2 (anima,x86_64)

template <u64_t PDP, u64_t PDS, u64_t PTS, bool free_list_p>
class pdpm_t
{
private:
  union free_pe_t
  {
    struct
      {
        u64_t next : 52;
        u64_t : 12;
      };
    u64_t raw;

    UTILS_RAW_ZERO (free_pe_t, u64_t)

    UTILS_SETTER (free_pe_t, u64_t, next)
  };

protected:
  static const u64_t PDP_PD_IDX = GBS (PDP & (GB (512) - 1));
  static const u64_t PDP_PT_IDX = MBS (PDP & (GB (1) - 1)) >> 1;
  static const u64_t PDP_PG_IDX = KBS (PDP & (MB (2) - 1)) >> 2;
  static const u64_t PDS_PD_IDX = GBS (PDS & (GB (512) - 1));
  static const u64_t PDS_PT_IDX = MBS (PDS & (GB (1) - 1)) >> 1;
  static const u64_t PTS_PD_IDX = GBS (PTS & (GB (512) - 1));

protected:
  static const u64_t PDP_PD = PDS + KB (4) * PDP_PD_IDX;
  static const u64_t PDP_PT = PTS + MB (2) * PDP_PD_IDX + KB (4) * PDP_PT_IDX;
  static const u64_t PDP_PTS_PT
    = PTS + MB (2) * PTS_PD_IDX + KB (4) * PDP_PD_IDX;
  static const u64_t PDS_PD = PDS + KB (4) * PDS_PD_IDX;
  static const u64_t PDS_PT = PTS + MB (2) * PDS_PD_IDX + KB (4) * PDS_PT_IDX;
  static const u64_t PDS_PTS_PT
    = PTS + MB (2) * PTS_PD_IDX + KB (4) * PDS_PD_IDX;
  static const u64_t PTS_PD = PDS + KB (4) * PTS_PD_IDX;
  static const u64_t PTS_PTS_PT
    = PTS + MB (2) * PTS_PD_IDX + KB (4) * PTS_PD_IDX;

private:
  free_pe_t *free_pe;
  spin_lock32_t free_pe_lock;

protected:
  bool
  init ()
  {
    if (PDP >= pdb::MAX_ADDR || PDS >= pdb::MAX_ADDR || PTS >= pdb::MAX_ADDR)
      return false;

    free_pe = NULL;
    return true;
  }

  void
  init_free_list (bool use_first_page_p)
  {
    using ::x86_64::pe_t;

    if (!free_list_p)
      return;

    // Link unused PDs.
    for (unsigned int i = 0; i < 512; ++i)
      {
        pe_t *pdpe = &((pe_t *) PDP)[i];

        if (!pdpe->P)
          add_free_pe (pdpe, true);
      }

    // Link unused PTs of PD(s) that contains PDP and PDS.
    if (PDP_PD_IDX == PDS_PD_IDX)
      {
        for (unsigned int i = 0; i < 512; ++i)
          if (i != PDP_PT_IDX && i != PDS_PT_IDX)
            add_free_pe (&((pe_t *) PDP_PD)[i], true);
      }
    else
      {
        for (unsigned int i = 0; i < 512; ++i)
          if (i != PDP_PT_IDX)
            add_free_pe (&((pe_t *) PDP_PD)[i], true);

        for (unsigned int i = 0; i < 512; ++i)
          if (i != PDS_PT_IDX)
            add_free_pe (&((pe_t *) PDS_PD)[i], true); 
      }

    // Link unused PTs.
    for (unsigned int i = 0; i < 512; ++i)
      {
        if (i == PDP_PD_IDX || i == PDS_PD_IDX || i == PTS_PD_IDX)
          continue;

        pe_t *pdpe = &((pe_t *) PDP)[i];

        if (!pdpe->P)
          continue;

        pe_t *pd = (pe_t *) (PDS + KB (4) * i);

        for (unsigned int j = 0; j < 512; ++j)
          {
            pe_t *pde = &pd[j];

            if (!pde->P)
              add_free_pe (pde, true);
          }
      }

    // Link unused pages of PT that contains PDP.
    for (unsigned int i = 0; i < 512; ++i)
      if (i != PDP_PG_IDX)
        add_free_pe (&((pe_t *) PDP_PT)[i], true);

    // Link unused pages.
    for (unsigned int i = 0; i < 512; ++i)
      {
        if (i == PDP_PD_IDX || i == PDS_PD_IDX || i == PTS_PD_IDX)
          continue;

        pe_t *pdpe = &((pe_t *) PDP)[i];

        if (!pdpe->P)
          continue;

        pe_t *pd = (pe_t *) (PDS + KB (4) * i);

        for (unsigned int j = 0; j < 512; ++j)
          {
            pe_t *pde = &pd[j];

            if (!pde->P)
              continue;

            pe_t *pt = (pe_t *) (PTS + MB (2) * i + KB (4) * j);

            for (unsigned int k
                   = (use_first_page_p && i == 0 && j == 0) ? 1 : 0;
                 k < 512; ++k)
              {
                pe_t *pte = &pt[k];

                if (!pte->P)
                  add_free_pe (pte, true);
              }
          }
      }
  }

  INLINE
  void
  add_free_pe (::x86_64::pe_t *pe, bool locked_p = false)
  {
    free_pe_lock.lock ();

    if (!locked_p)
      {
        mm::wlock (*pe);

        if (pe->P)
          {
            mm::unwlock (*pe);
            return;
          }
      }

    ((free_pe_t *) pe)->set_next ((u64_t) free_pe);
    free_pe = (free_pe_t *) pe;

    if (!locked_p)
      mm::unwlock (*pe);
      
    free_pe_lock.unlock ();
  }

  INLINE
  ::x86_64::vaddr_t
  get_free_addr (::x86_64::pe_t *&pe)
  {
    u64_t addr;

    free_pe_lock.lock ();

    if (!free_pe)
      {
        free_pe_lock.unlock ();
        return 0;
      }

    pe = (::x86_64::pe_t *) free_pe;
    free_pe = (free_pe_t *) free_pe->next;

    free_pe_lock.unlock ();

    if ((u64_t) pe >= PDP && (u64_t) pe < PDP + 512)
      addr = ((u64_t) pe - PDP) * GB (1);
    else if ((u64_t) pe >= PDS && (u64_t) pe < PDS + MB (2))
      addr = (KBS ((u64_t) pe - PDS) >> 2) * GB (1)
             + (((u64_t) pe - CUT ((u64_t) pe, KB (4))) >> 3) * MB (2);
    else
      addr = (MBS ((u64_t) pe - PTS) >> 1) * GB (1)
             + (KBS ((u64_t) pe - CUT ((u64_t) pe, MB (2))) >> 2) * MB (2)
             + (((u64_t) pe - CUT ((u64_t) pe, KB (4))) >> 3) * KB (4);
    
    return addr;
  }

  INLINE
  ::x86_64::vaddr_t
  check_addr (u64_t addr)
  {
    ::x86_64::vaddr_t vaddr = CUT (addr, 4096);

    if (vaddr == PDP
        || vaddr >= PDS && vaddr < PDS + MB (2)
        || vaddr >= PTS && vaddr < PTS + GB (1)
        || vaddr == 0)
      return 0;

    return vaddr;
  }

public:
  mm::tte_t *
  get_pte (u64_t addr)
  {
    using ::x86_64::pe_t;

    ::x86_64::vaddr_t vaddr = check_addr (addr);

    if (vaddr == 0)
      return NULL;

    pe_t *pdpe = &((pe_t *) PDP)[vaddr.PDP];
    pe_t *pde = &((pe_t *) (PDS + KB (4) * vaddr.PDP))[vaddr.PD];
    mm::tte_t *pte
      = &((mm::tte_t *) (PTS + MB (2) * vaddr.PDP
                         + KB (4) * vaddr.PD))[vaddr.PT];

    mm::rlock (*pdpe);

    if (!pdpe->P)
      {
        mm::unrlock (*pdpe);
        return NULL;
      }

    mm::unrlock (*pdpe);

    mm::rlock (*pde);

    if (!pde->P)
      {
        mm::unrlock (*pde);
        return NULL;
      }

    mm::unrlock (*pde);

    return pte;
  }

  mm::tte_t
  unmap (mm::tte_t *tte)
  {
    if (free_list_p || (u64_t) tte < PTS || (u64_t) tte >= PTS)
      return false;

    tte->wlock ();

    mm::tte_t result = *tte;

    tte->set_P (false);

    tte->unwlock ();

    return result;
  }

  mm::tte_t
  unmap (u64_t addr)
  {
    mm::tte_t *tte = get_pte (addr);

    if (!tte)
      return 0;

    return unmap (tte);
  }
};

template <u64_t PDP, u64_t PDS, u64_t PTS, typename ALLOCATOR,
          bool free_list_p>
class self_pdpm_t : public pdpm_t<PDP, PDS, PTS, free_list_p>
{
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PD_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PT_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PG_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PD_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PT_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PTS_PD_IDX;

  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PD;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PD;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PTS_PD;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PTS_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PTS_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PTS_PTS_PT;

  using pdpm_t<PDP, PDS, PTS, free_list_p>::add_free_pe;

private:
  ALLOCATOR *allocator;

public:
  bool
  init (ALLOCATOR *allocator, ::x86_64::pe_t pml4e)
  {
    using ::x86_64::pe_t;
    using ::anima::pdb;

    if (!pdpm_t<PDP, PDS, PTS, free_list_p>::init ())
      return false;

    this->allocator = allocator;

    pdb::lvl1e_t *pdp_lvl1e = NULL;
    
    if (!pml4e.P)
      {
        pdp_lvl1e = allocator->allocate ();

        if (!pdp_lvl1e)
          return false;
      }

    pdb::lvl1e_t *pdp_pd_lvl1e = allocator->allocate ();

    if (!pdp_pd_lvl1e)
      {
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    pdb::lvl1e_t *pdp_pt_lvl1e = allocator->allocate ();

    if (!pdp_pt_lvl1e)
      {
        allocator->deallocate (pdp_pd_lvl1e);
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    pdb::lvl1e_t *pds_pd_lvl1e = NULL;
    
    if (PDS_PD_IDX == PDP_PD_IDX)
      pds_pd_lvl1e = pdp_pd_lvl1e;
    else
      pds_pd_lvl1e = allocator->allocate ();
    
    if (!pds_pd_lvl1e)
      {
        allocator->deallocate (pdp_pt_lvl1e);
        allocator->deallocate (pdp_pd_lvl1e);
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    pdb::lvl1e_t *pds_pt_lvl1e = allocator->allocate ();

    if (!pds_pt_lvl1e)
      {
        allocator->deallocate (pds_pd_lvl1e);
        allocator->deallocate (pdp_pt_lvl1e);
        allocator->deallocate (pdp_pd_lvl1e);
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    pdb::lvl1e_t *pts_pd_lvl1e = allocator->allocate ();

    if (!pts_pd_lvl1e)
      {
        allocator->deallocate (pds_pt_lvl1e);
        allocator->deallocate (pds_pd_lvl1e);
        allocator->deallocate (pdp_pt_lvl1e);
        allocator->deallocate (pdp_pd_lvl1e);
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    pdb::lvl1e_t *pdp_pts_pt_lvl1e = allocator->allocate ();

    if (!pdp_pts_pt_lvl1e)
      {
        allocator->deallocate (pts_pd_lvl1e);
        allocator->deallocate (pds_pd_lvl1e);
        allocator->deallocate (pds_pt_lvl1e);
        allocator->deallocate (pdp_pt_lvl1e);
        allocator->deallocate (pdp_pd_lvl1e);
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    pdb::lvl1e_t *pds_pts_pt_lvl1e = NULL;

    if (PDS_PD_IDX == PDP_PD_IDX)
      pds_pts_pt_lvl1e = pdp_pts_pt_lvl1e;
    else
      pds_pts_pt_lvl1e = allocator->allocate ();

    if (!pds_pts_pt_lvl1e)
      {
        allocator->deallocate (pdp_pts_pt_lvl1e);
        allocator->deallocate (pts_pd_lvl1e);
        allocator->deallocate (pds_pd_lvl1e);
        allocator->deallocate (pds_pt_lvl1e);
        allocator->deallocate (pdp_pt_lvl1e);
        allocator->deallocate (pdp_pd_lvl1e);
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    pdb::lvl1e_t *pts_pts_pt_lvl1e = allocator->allocate ();

    if (!pts_pts_pt_lvl1e)
      {
        allocator->deallocate (pds_pts_pt_lvl1e);
        allocator->deallocate (pdp_pts_pt_lvl1e);
        allocator->deallocate (pts_pd_lvl1e);
        allocator->deallocate (pds_pd_lvl1e);
        allocator->deallocate (pds_pt_lvl1e);
        allocator->deallocate (pdp_pt_lvl1e);
        allocator->deallocate (pdp_pd_lvl1e);
        allocator->deallocate (pdp_lvl1e);
        return false;
      }

    u64_t pdp_paddr
      = pdp_lvl1e ? pdp_lvl1e->entry.get_ptr () : pml4e.get_ptr ();

    pml4e.set_raw (0).set_ptr (pdp_paddr).set_RW ().set_P ();
    allocator->install (pml4e);

    pe_t *tmp = allocator->map (pdp_paddr);

    if (pdp_lvl1e)
      utils::zero (tmp, KB (4));

    tmp[PDP_PD_IDX].set_base (pdp_pd_lvl1e->entry.base)
      .set_RW ().set_P ();
    tmp[PDS_PD_IDX].set_base (pds_pd_lvl1e->entry.base)
      .set_RW ().set_P ();
    tmp[PTS_PD_IDX].set_base (pts_pd_lvl1e->entry.base)
      .set_RW ().set_P ();

    allocator->unmap ();

    tmp = allocator->map (pdp_pd_lvl1e->entry.get_ptr ());
    utils::zero (tmp, KB (4));

    tmp[PDP_PT_IDX].set_base (pdp_pt_lvl1e->entry.base)
      .set_RW ().set_P ();

    if (PDS_PD_IDX != PDP_PD_IDX)
      {
        allocator->unmap ();
        tmp = allocator->map (pds_pd_lvl1e->entry.get_ptr ());
        utils::zero (tmp, KB (4));
      }

    tmp[PDS_PT_IDX].set_base (pds_pt_lvl1e->entry.base)
      .set_RW ().set_P ();

    allocator->unmap ();

    tmp = allocator->map (pds_pt_lvl1e->entry.get_ptr ());
    utils::zero (tmp, KB (4));

    ((pe_t *) tmp)[PDP_PD_IDX].set_base (pdp_pd_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) tmp)[PDS_PD_IDX].set_base (pds_pd_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) tmp)[PTS_PD_IDX].set_base (pts_pd_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();

    allocator->unmap ();

    utils::zero ((void *) PTS_PD, KB (4));

    ((pe_t *) PTS_PD)[PDP_PD_IDX].set_base (pdp_pts_pt_lvl1e->entry.base)
      .set_RW ().set_P ();
    ((pe_t *) PTS_PD)[PDS_PD_IDX].set_base (pds_pts_pt_lvl1e->entry.base)
      .set_RW ().set_P ();
    ((pe_t *) PTS_PD)[PTS_PD_IDX].set_base (pts_pts_pt_lvl1e->entry.base)
      .set_RW ().set_P ();

    tmp = allocator->map (pts_pts_pt_lvl1e->entry.get_ptr ());
    utils::zero (tmp, KB (4));

    ((pe_t *) tmp)[PDP_PD_IDX].set_base (pdp_pts_pt_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) tmp)[PDS_PD_IDX].set_base (pds_pts_pt_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) tmp)[PTS_PD_IDX].set_base (pts_pts_pt_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();

    allocator->unmap ();

    utils::zero ((void *) PDP_PTS_PT, KB (4));

    if (PDS_PD_IDX != PDP_PD_IDX)
      utils::zero ((void *) PDS_PTS_PT, KB (4));

    ((pe_t *) PDP_PTS_PT)[PDP_PT_IDX].set_base (pdp_pt_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) PDS_PTS_PT)[PDS_PT_IDX].set_base (pds_pt_lvl1e->entry.base)
      .set_NX ().set_RW ().set_P ();

    utils::zero ((void *) PDP_PT, KB (4));

    ((pe_t *) PDP_PT)[PDP_PG_IDX].set_ptr (pdp_paddr)
      .set_NX ().set_RW ().set_P ();

    if (pdp_lvl1e)
      return true;

    // Map used PDs and PTs.
    for (unsigned int i = 0; i < 512; ++i)
      {
        if (i == PDP_PD_IDX || i == PDS_PD_IDX || i == PTS_PD_IDX)
          continue;

        pe_t *pdpe = &((pe_t *) PDP)[i];

        if (!pdpe->P)
          continue;

        ((pe_t *) PDS_PT)[i].set_base (pdpe->base)
          .set_NX ().set_RW ().set_P ();

        pdb::lvl1e_t *pts_pt_lvl1e = allocator->allocate ();

        if (!pts_pt_lvl1e)
          return false;

        ((pe_t *) PTS_PD)[i].set_base (pts_pt_lvl1e->entry.base)
          .set_RW ().set_P ();
        ((pe_t *) PTS_PTS_PT)[i].set_base (pts_pt_lvl1e->entry.base)
          .set_NX ().set_RW ().set_P ();

        pe_t *pd = (pe_t *) (PDS + KB (4) * i);
        pe_t *pts_pt = (pe_t *) (PTS + MB (2) * PTS_PD_IDX + KB (4) * i);

        for (unsigned int j = 0; j < 512; ++j)
          {
            if (pd[j].P)
              pts_pt[j].set_base (pd[j].base).set_NX ().set_RW ().set_P ();
          }
      }

    pdpm_t<PDP, PDS, PTS, free_list_p>::init_free_list (
      ((::x86_64::vaddr_t) PDP).PML4 != 0);

    return true;
  }

  bool
  init (ALLOCATOR *allocator)
  {
    return init (allocator, (::x86_64::pe_t) 0);
  }

private:
  mm::tte_t *
  map (::x86_64::vaddr_t vaddr)
  {
    using ::x86_64::pe_t;
    using ::anima::pdb;

    pe_t *pdpe = &((pe_t *) PDP)[vaddr.PDP];
    pe_t *pd = (pe_t *) (PDS + KB (4) * vaddr.PDP);
    pe_t *pts_pt = (pe_t *) (PTS + MB (2) * PTS_PD_IDX + KB (4) * vaddr.PDP);

    mm::rlock (*pdpe);

    if (!pdpe->P && mm::wlock_if_np (*pdpe))
      {
        pdb::lvl1e_t *pd_lvl1e = allocator->allocate ();

        if (!pd_lvl1e)
          {
            mm::unwlock (*pdpe);
            return NULL;
          }

        pdb::lvl1e_t *pts_pt_lvl1e = allocator->allocate ();

        if (!pts_pt_lvl1e)
          {
            mm::unwlock (*pdpe);
            allocator->deallocate (pd_lvl1e);
            return NULL;
          }

        ((pe_t *) PDS_PT)[vaddr.PDP].set_base (pd_lvl1e->entry.base)
          .set_NX ().set_RW ().set_P ();
        ((pe_t *) PTS_PD)[vaddr.PDP].set_base (pts_pt_lvl1e->entry.base)
          .set_RW ().set_P ();
        ((pe_t *) PTS_PTS_PT)[vaddr.PDP].set_base (pts_pt_lvl1e->entry.base)
          .set_NX ().set_RW ().set_P ();

        mm::clear (*pdpe).set_base (pd_lvl1e->entry.base)
          .set_US ().set_RW ().set_P ();

        utils::zero (pts_pt, KB (4));

        if (free_list_p)
          {
            for (unsigned int i = 0; i < 512; ++i)
              if (i == vaddr.PD)
                pd[i] = 0;
              else
                add_free_pe (&pd[i], true);
          }
        else
          utils::zero (pd, KB (4));

        mm::unwlock (*pdpe);
      }
    else
     mm::unrlock (*pdpe);

    pe_t *pde = &pd[vaddr.PD];
    mm::tte_t *pt
      = (mm::tte_t *) (PTS + MB (2) * vaddr.PDP + KB (4) * vaddr.PD);

    mm::rlock (*pde);

    if (!pde->P && mm::wlock_if_np (*pde))
      {
        pdb::lvl1e_t *pt_lvl1e = allocator->allocate ();

        if (!pt_lvl1e)
          {
            mm::unwlock (*pde);
            return NULL;
          }

        pts_pt[vaddr.PD].set_base (pt_lvl1e->entry.base)
          .set_NX ().set_RW ().set_P ();
        mm::clear (*pde).set_base (pt_lvl1e->entry.base)
          .set_US ().set_RW ().set_P ();

        if (free_list_p)
          {
            for (unsigned int i = 0; i < 512; ++i)
              if (i == vaddr.PT)
                pt[i] = 0;
              else
                add_free_pe (&pt[i], true);
          }
        else
          utils::zero (pt, KB (4));

        mm::unwlock (*pde);
      }
    else
      mm::unrlock (*pde);

    return &pt[vaddr.PT];
  }

public:
  mm::tte_t *
  map_free (u64_t &addr)
  {
    if (!free_list_p)
      return NULL;

    ::x86_64::pe_t *old_free_pe = NULL;
    ::x86_64::vaddr_t vaddr
      = pdpm_t<PDP, PDS, PTS, free_list_p>::get_free_addr (old_free_pe);

    if (vaddr == 0)
      return NULL;

    mm::tte_t *tte = map (vaddr);

    if (!tte)
      {
        add_free_pe (old_free_pe);
        return NULL;
      }

    addr = vaddr;
    return tte;
  }

  mm::tte_t *
  map (u64_t addr)
  {
    if (free_list_p)
      return NULL;

    ::x86_64::vaddr_t vaddr =
      pdpm_t<PDP, PDS, PTS, free_list_p>::check_addr (addr);

    if (vaddr == 0)
      return NULL;

    return map (vaddr);
  }
};

template <u64_t PDP, u64_t PDS, u64_t PTS, typename MAPPER, bool free_list_p>
class ext_pdpm_t : public pdpm_t<PDP, PDS, PTS, free_list_p>
{
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PD_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PT_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PG_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PD_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PT_IDX;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PTS_PD_IDX;

  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PD;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PD;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PTS_PD;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDP_PTS_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PDS_PTS_PT;
  using pdpm_t<PDP, PDS, PTS, free_list_p>::PTS_PTS_PT;

  using pdpm_t<PDP, PDS, PTS, free_list_p>::add_free_pe;

private:
  MAPPER *mapper;

public:
  ::x86_64::pe_t
  init (MAPPER *mapper)
  {
    using ::x86_64::pe_t;

    if (!pdpm_t<PDP, PDS, PTS, free_list_p>::init ())
      return false;

    this->mapper = mapper;

    mm::tte_t *pdp_pte = mapper->map (PDP);

    if (!pdp_pte)
      return false;

    mm::tte_t *pdp_pd_pte = mapper->map (PDP_PD);

    if (!pdp_pd_pte)
      {
        mapper->unmap (pdp_pte);
        return false;
      }

    mm::tte_t *pdp_pt_pte = mapper->map (PDP_PT);

    if (!pdp_pt_pte)
      {
        mapper->unmap (pdp_pd_pte);
        mapper->unmap (pdp_pte);
        return false;
      }

    mm::tte_t *pds_pd_pte = NULL;
    
    if (PDS_PD_IDX == PDP_PD_IDX)
      pds_pd_pte = pdp_pd_pte;
    else
      pds_pd_pte = mapper->map (PDS_PD);

    if (!pds_pd_pte)
      {
        mapper->unmap (pdp_pt_pte);
        mapper->unmap (pdp_pd_pte);
        mapper->unmap (pdp_pte);
        return false;
      }

    mm::tte_t *pds_pt_pte = mapper->map (PDS_PT);

    if (!pds_pt_pte)
      {
        mapper->unmap (pds_pd_pte);
        mapper->unmap (pdp_pt_pte);
        mapper->unmap (pdp_pd_pte);
        mapper->unmap (pdp_pte);
        return false;
      }

    mm::tte_t *pts_pd_pte = mapper->map (PTS_PD);

    if (!pts_pd_pte)
      {
        mapper->unmap (pds_pt_pte);
        mapper->unmap (pds_pd_pte);
        mapper->unmap (pdp_pt_pte);
        mapper->unmap (pdp_pd_pte);
        mapper->unmap (pdp_pte);
        return false;
      }

    mm::tte_t *pdp_pts_pt_pte = mapper->map (PDP_PTS_PT);

    if (!pdp_pts_pt_pte)
      {
        mapper->unmap (pts_pd_pte);
        mapper->unmap (pds_pt_pte);
        mapper->unmap (pds_pd_pte);
        mapper->unmap (pdp_pt_pte);
        mapper->unmap (pdp_pd_pte);
        mapper->unmap (pdp_pte);
        return false;
      }

    mm::tte_t *pds_pts_pt_pte = NULL;
    
    if (PDS_PD_IDX == PDP_PD_IDX)
      pds_pts_pt_pte = pdp_pts_pt_pte;
    else
      pds_pts_pt_pte = mapper->map (PDS_PTS_PT);

    if (!pds_pts_pt_pte)
      {
        mapper->unmap (pdp_pts_pt_pte);
        mapper->unmap (pts_pd_pte);
        mapper->unmap (pds_pt_pte);
        mapper->unmap (pds_pd_pte);
        mapper->unmap (pdp_pt_pte);
        mapper->unmap (pdp_pd_pte);
        mapper->unmap (pdp_pte);
        return false;
      }

    mm::tte_t *pts_pts_pt_pte = mapper->map (PTS_PTS_PT);

    if (!pts_pts_pt_pte)
      {
        mapper->unmap (pts_pts_pt_pte);
        mapper->unmap (pdp_pts_pt_pte);
        mapper->unmap (pts_pd_pte);
        mapper->unmap (pds_pt_pte);
        mapper->unmap (pds_pd_pte);
        mapper->unmap (pdp_pt_pte);
        mapper->unmap (pdp_pd_pte);
        mapper->unmap (pdp_pte);
        return false;
      }

    utils::zero ((void *) PDP, KB (4));
    utils::zero ((void *) PDP_PD, KB (4));
    utils::zero ((void *) PDS_PD, KB (4));
    utils::zero ((void *) PTS_PD, KB (4));
    utils::zero ((void *) PDP_PTS_PT, KB (4));
    utils::zero ((void *) PDS_PTS_PT, KB (4));
    utils::zero ((void *) PTS_PTS_PT, KB (4));
    utils::zero ((void *) PDP_PT, KB (4));
    utils::zero ((void *) PDS_PT, KB (4));

    ((pe_t *) PDP)[PDP_PD_IDX].set_base (pdp_pd_pte->base).set_RW ().set_P ();
    ((pe_t *) PDP)[PDS_PD_IDX].set_base (pds_pd_pte->base).set_RW ().set_P ();
    ((pe_t *) PDP)[PTS_PD_IDX].set_base (pts_pd_pte->base).set_RW ().set_P ();

    ((pe_t *) PDP_PD)[PDP_PT_IDX].set_base (pdp_pt_pte->base)
      .set_RW ().set_P ();

    ((pe_t *) PDS_PD)[PDS_PT_IDX].set_base (pds_pt_pte->base)
      .set_RW ().set_P ();

    ((pe_t *) PDS_PT)[PDP_PD_IDX].set_base (pdp_pd_pte->base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) PDS_PT)[PDS_PD_IDX].set_base (pds_pd_pte->base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) PDS_PT)[PTS_PD_IDX].set_base (pts_pd_pte->base)
      .set_NX ().set_RW ().set_P ();

    ((pe_t *) PTS_PD)[PDP_PD_IDX].set_base (pdp_pts_pt_pte->base)
      .set_RW ().set_P ();
    ((pe_t *) PTS_PD)[PDS_PD_IDX].set_base (pds_pts_pt_pte->base)
      .set_RW ().set_P ();
    ((pe_t *) PTS_PD)[PTS_PD_IDX].set_base (pts_pts_pt_pte->base)
      .set_RW ().set_P ();

    ((pe_t *) PTS_PTS_PT)[PDP_PD_IDX].set_base (pdp_pts_pt_pte->base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) PTS_PTS_PT)[PDS_PD_IDX].set_base (pds_pts_pt_pte->base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) PTS_PTS_PT)[PTS_PD_IDX].set_base (pts_pts_pt_pte->base)
      .set_NX ().set_RW ().set_P ();

    ((pe_t *) PDP_PTS_PT)[PDP_PT_IDX].set_base (pdp_pt_pte->base)
      .set_NX ().set_RW ().set_P ();
    ((pe_t *) PDS_PTS_PT)[PDS_PT_IDX].set_base (pds_pt_pte->base)
      .set_NX ().set_RW ().set_P ();

    ((pe_t *) PDP_PT)[PDP_PG_IDX].set_base (pdp_pte->base)
      .set_NX ().set_RW ().set_P ();

    pdpm_t<PDP, PDS, PTS, free_list_p>::init_free_list (true);

    return pe_t ().set_base (pdp_pte->base).set_P ();
  }

private:
  mm::tte_t *
  map (::x86_64::vaddr_t vaddr)
  {
    using ::x86_64::pe_t;

    pe_t *pdpe = &((pe_t *) PDP)[vaddr.PDP];
    pe_t *pd = (pe_t *) (PDS + KB (4) * vaddr.PDP);

    mm::rlock (*pdpe);

    if (!pdpe->P && mm::wlock_if_np (*pdpe))
      {
        pe_t *pd_pte = mapper->map ((u64_t) pd);

        if (!pd_pte)
          {
            mm::unwlock (*pdpe);
            return NULL;
          }

        mm::clear (*pdpe).set_base (pd_pte->base).set_US ().set_RW ().set_P ();

        if (free_list_p)
          {
            for (unsigned int i = 0; i < 512; ++i)
              if (i != vaddr.PD)
                add_free_pe (&pd[i], true);
          }
        else
          utils::zero (pd, KB (4));

        mm::unwlock (*pdpe);
      }
    else
      mm::unrlock (*pdpe);

    pe_t *pde = &((pe_t *) (PDS + KB (4) * vaddr.PDP))[vaddr.PD];
    mm::tte_t *pt
      = (mm::tte_t *) (PTS + MB (2) * vaddr.PDP + KB (4) * vaddr.PD);

    mm::rlock (*pde);

    if (!pde->P && mm::wlock_if_np (*pde))
      {
        pe_t *pt_pte = mapper->map ((u64_t) pt);

        if (!pt_pte)
          {
            mm::unwlock (*pde);
            return NULL;
          }

        mm::clear (*pde).set_base (pt_pte->base).set_US ().set_RW ().set_P ();

        if (free_list_p)
          {
            for (unsigned int i = 0; i < 512; ++i)
              if (i != vaddr.PT)
                add_free_pe (&pt[i], true);
          }
        else
          utils::zero (pt, KB (4));

        mm::unwlock (*pde);
      }
    else
      mm::unrlock (*pde);

    return &pt[vaddr.PT];
  }

public:
  mm::tte_t *
  map_free (u64_t &addr)
  {
    if (!free_list_p)
      return NULL;

    ::x86_64::pe_t *old_free_pe = NULL;
    ::x86_64::vaddr_t vaddr
      = pdpm_t<PDP, PDS, PTS, free_list_p>::get_free_addr (old_free_pe);

    if (vaddr == 0)
      return NULL;

    mm::tte_t *tte = map (vaddr);

    if (!tte)
      {
        add_free_pe (old_free_pe);
        return NULL;
      }

    addr = vaddr;
    return tte;
  }

  mm::tte_t *
  map (u64_t addr)
  {
    if (free_list_p)
      return NULL;

    ::x86_64::vaddr_t vaddr
      = pdpm_t<PDP, PDS, PTS, free_list_p>::check_addr (addr);

    if (vaddr == 0)
      return NULL;

    return map (vaddr);
  }
};

END_NAMESPACE2

#endif /* HEADER_ANIMA_X86_64_PDPM_H */

